{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "matchzoo version 2.1.0\n",
      "\n",
      "data loading ...\n",
      "data loaded as `train_pack_raw` `dev_pack_raw` `test_pack_raw`\n",
      "`ranking_task` initialized with metrics [normalized_discounted_cumulative_gain@3(0.0), normalized_discounted_cumulative_gain@5(0.0), mean_average_precision(0.0)]\n",
      "loading embedding ...\n",
      "embedding loaded as `glove_embedding`\n"
     ]
    }
   ],
   "source": [
    "%run init.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessor = mz.preprocessors.BasicPreprocessor(fixed_length_left=10, \n",
    "                                                  fixed_length_right=40, \n",
    "                                                  remove_stop_words=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval => StopRemoval: 100%|██████████| 2118/2118 [00:00<00:00, 8302.01it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval => StopRemoval: 100%|██████████| 18841/18841 [00:04<00:00, 4319.36it/s]\n",
      "Processing text_right with append: 100%|██████████| 18841/18841 [00:00<00:00, 903668.21it/s]\n",
      "Building FrequencyFilter from a datapack.: 100%|██████████| 18841/18841 [00:00<00:00, 195422.81it/s]\n",
      "Processing text_right with transform: 100%|██████████| 18841/18841 [00:00<00:00, 108320.04it/s]\n",
      "Processing text_left with extend: 100%|██████████| 2118/2118 [00:00<00:00, 725009.05it/s]\n",
      "Processing text_right with extend: 100%|██████████| 18841/18841 [00:00<00:00, 860396.98it/s]\n",
      "Building Vocabulary from a datapack.: 100%|██████████| 234263/234263 [00:00<00:00, 2861207.82it/s]\n",
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval => StopRemoval: 100%|██████████| 2118/2118 [00:00<00:00, 8464.35it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval => StopRemoval: 100%|██████████| 18841/18841 [00:04<00:00, 4340.04it/s]\n",
      "Processing text_right with transform: 100%|██████████| 18841/18841 [00:00<00:00, 202575.96it/s]\n",
      "Processing text_left with transform: 100%|██████████| 2118/2118 [00:00<00:00, 308616.84it/s]\n",
      "Processing text_right with transform: 100%|██████████| 18841/18841 [00:00<00:00, 195001.35it/s]\n",
      "Processing length_left with len: 100%|██████████| 2118/2118 [00:00<00:00, 660829.86it/s]\n",
      "Processing length_right with len: 100%|██████████| 18841/18841 [00:00<00:00, 823132.98it/s]\n",
      "Processing text_left with transform: 100%|██████████| 2118/2118 [00:00<00:00, 144222.61it/s]\n",
      "Processing text_right with transform: 100%|██████████| 18841/18841 [00:00<00:00, 116667.55it/s]\n",
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval => StopRemoval: 100%|██████████| 122/122 [00:00<00:00, 8410.53it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval => StopRemoval: 100%|██████████| 1115/1115 [00:00<00:00, 4361.18it/s]\n",
      "Processing text_right with transform: 100%|██████████| 1115/1115 [00:00<00:00, 10806.91it/s]\n",
      "Processing text_left with transform: 100%|██████████| 122/122 [00:00<00:00, 142655.45it/s]\n",
      "Processing text_right with transform: 100%|██████████| 1115/1115 [00:00<00:00, 186424.66it/s]\n",
      "Processing length_left with len: 100%|██████████| 122/122 [00:00<00:00, 193096.26it/s]\n",
      "Processing length_right with len: 100%|██████████| 1115/1115 [00:00<00:00, 585386.03it/s]\n",
      "Processing text_left with transform: 100%|██████████| 122/122 [00:00<00:00, 88853.11it/s]\n",
      "Processing text_right with transform: 100%|██████████| 1115/1115 [00:00<00:00, 114908.20it/s]\n",
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval => StopRemoval: 100%|██████████| 237/237 [00:00<00:00, 8602.77it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval => StopRemoval: 100%|██████████| 2300/2300 [00:00<00:00, 4324.46it/s]\n",
      "Processing text_right with transform: 100%|██████████| 2300/2300 [00:00<00:00, 208270.89it/s]\n",
      "Processing text_left with transform: 100%|██████████| 237/237 [00:00<00:00, 242925.23it/s]\n",
      "Processing text_right with transform: 100%|██████████| 2300/2300 [00:00<00:00, 199443.84it/s]\n",
      "Processing length_left with len: 100%|██████████| 237/237 [00:00<00:00, 323689.37it/s]\n",
      "Processing length_right with len: 100%|██████████| 2300/2300 [00:00<00:00, 703075.52it/s]\n",
      "Processing text_left with transform: 100%|██████████| 237/237 [00:00<00:00, 113696.68it/s]\n",
      "Processing text_right with transform: 100%|██████████| 2300/2300 [00:00<00:00, 119756.92it/s]\n"
     ]
    }
   ],
   "source": [
    "train_pack_processed = preprocessor.fit_transform(train_pack_raw)\n",
    "dev_pack_processed = preprocessor.transform(dev_pack_raw)\n",
    "test_pack_processed = preprocessor.transform(test_pack_raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model_class                   <class 'matchzoo.models.match_pyramid.MatchPyramid'>\n",
      "input_shapes                  [(10,), (40,)]\n",
      "task                          Ranking Task\n",
      "optimizer                     adam\n",
      "with_embedding                True\n",
      "embedding_input_dim           16546\n",
      "embedding_output_dim          100\n",
      "embedding_trainable           True\n",
      "num_blocks                    2\n",
      "kernel_count                  [16, 32]\n",
      "kernel_size                   [[3, 3], [3, 3]]\n",
      "activation                    relu\n",
      "dpool_size                    [3, 10]\n",
      "padding                       same\n",
      "dropout_rate                  0.1\n"
     ]
    }
   ],
   "source": [
    "model = mz.models.MatchPyramid()\n",
    "\n",
    "# load `input_shapes` and `embedding_input_dim` (vocab_size)\n",
    "model.params.update(preprocessor.context)\n",
    "\n",
    "model.params['task'] = ranking_task\n",
    "model.params['embedding_output_dim'] = 100\n",
    "model.params['embedding_trainable'] = True\n",
    "model.params['num_blocks'] = 2\n",
    "model.params['kernel_count'] = [16, 32]\n",
    "model.params['kernel_size'] = [[3, 3], [3, 3]]\n",
    "model.params['dpool_size'] = [3, 10]\n",
    "model.params['optimizer'] = 'adam'\n",
    "model.params['dropout_rate'] = 0.1\n",
    "\n",
    "model.build()\n",
    "model.compile()\n",
    "\n",
    "print(model.params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "text_left (InputLayer)          (None, 10)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "text_right (InputLayer)         (None, 40)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           multiple             1654600     text_left[0][0]                  \n",
      "                                                                 text_right[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "matching_layer_1 (MatchingLayer (None, 10, 40, 1)    0           embedding[0][0]                  \n",
      "                                                                 embedding[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_1 (Conv2D)               (None, 10, 40, 16)   160         matching_layer_1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_2 (Conv2D)               (None, 10, 40, 32)   4640        conv2d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dpool_index (InputLayer)        (None, 10, 40, 2)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dynamic_pooling_layer_1 (Dynami (None, 3, 10, 32)    0           conv2d_2[0][0]                   \n",
      "                                                                 dpool_index[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 960)          0           dynamic_pooling_layer_1[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 960)          0           flatten_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 1)            961         dropout_1[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 1,660,361\n",
      "Trainable params: 1,660,361\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.backend.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_matrix = glove_embedding.build_matrix(preprocessor.context['vocab_unit'].state['term_index'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_embedding_matrix(embedding_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num batches: 102\n"
     ]
    }
   ],
   "source": [
    "dpool_callback = mz.data_generator.callbacks.DynamicPooling(\n",
    "    fixed_length_left=10, \n",
    "    fixed_length_right=40\n",
    ")\n",
    "train_generator = mz.DataGenerator(\n",
    "    train_pack_processed,\n",
    "    mode='pair',\n",
    "    num_dup=2,\n",
    "    num_neg=1,\n",
    "    batch_size=20,\n",
    "    callbacks=[dpool_callback]\n",
    ")\n",
    "print('num batches:', len(train_generator))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "118"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_generator = mz.DataGenerator(\n",
    "    test_pack_processed,\n",
    "    batch_size=20,\n",
    "    callbacks=[dpool_callback]\n",
    ")\n",
    "len(test_generator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_x, test_y = test_generator[:]\n",
    "evaluate = mz.callbacks.EvaluateAllMetrics(model, x=test_x, y=test_y, batch_size=len(test_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "102/102 [==============================] - 5s 47ms/step - loss: 0.8098\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5543583477044723 - normalized_discounted_cumulative_gain@5(0.0): 0.6116356675515685 - mean_average_precision(0.0): 0.5743161762170562\n",
      "Epoch 2/20\n",
      "102/102 [==============================] - 10s 95ms/step - loss: 0.5260\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5301739579616542 - normalized_discounted_cumulative_gain@5(0.0): 0.6016159230710824 - mean_average_precision(0.0): 0.5601950549545589\n",
      "Epoch 3/20\n",
      "102/102 [==============================] - 10s 101ms/step - loss: 0.3829\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5092929494151631 - normalized_discounted_cumulative_gain@5(0.0): 0.591478646606495 - mean_average_precision(0.0): 0.5441527086702361\n",
      "Epoch 4/20\n",
      "102/102 [==============================] - 10s 96ms/step - loss: 0.3304\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.4812004958713687 - normalized_discounted_cumulative_gain@5(0.0): 0.5596264451275563 - mean_average_precision(0.0): 0.5088711984093643\n",
      "Epoch 5/20\n",
      "102/102 [==============================] - 11s 109ms/step - loss: 0.2227\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.508961151215818 - normalized_discounted_cumulative_gain@5(0.0): 0.5842189791253107 - mean_average_precision(0.0): 0.5374359920620746\n",
      "Epoch 6/20\n",
      "102/102 [==============================] - 11s 108ms/step - loss: 0.1847\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.4934803343084 - normalized_discounted_cumulative_gain@5(0.0): 0.5644795486607259 - mean_average_precision(0.0): 0.5198714294860981\n",
      "Epoch 7/20\n",
      "102/102 [==============================] - 11s 108ms/step - loss: 0.1371\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5107068648352034 - normalized_discounted_cumulative_gain@5(0.0): 0.5784800117656429 - mean_average_precision(0.0): 0.5308213418764718\n",
      "Epoch 8/20\n",
      "102/102 [==============================] - 10s 97ms/step - loss: 0.1038\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.506959204392066 - normalized_discounted_cumulative_gain@5(0.0): 0.5816987880270651 - mean_average_precision(0.0): 0.5314363380052429\n",
      "Epoch 9/20\n",
      "102/102 [==============================] - 10s 95ms/step - loss: 0.0863\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.49359062647045354 - normalized_discounted_cumulative_gain@5(0.0): 0.5634597428623797 - mean_average_precision(0.0): 0.5230596215775214\n",
      "Epoch 10/20\n",
      "102/102 [==============================] - 10s 99ms/step - loss: 0.0791\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.514243097562575 - normalized_discounted_cumulative_gain@5(0.0): 0.575757696009702 - mean_average_precision(0.0): 0.5351664532621346\n",
      "Epoch 11/20\n",
      "102/102 [==============================] - 11s 110ms/step - loss: 0.0486\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5149547148083139 - normalized_discounted_cumulative_gain@5(0.0): 0.5730596976101693 - mean_average_precision(0.0): 0.533951748706618\n",
      "Epoch 12/20\n",
      "102/102 [==============================] - 11s 108ms/step - loss: 0.0417\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5089291208562943 - normalized_discounted_cumulative_gain@5(0.0): 0.5729989997270044 - mean_average_precision(0.0): 0.5339051698603664\n",
      "Epoch 13/20\n",
      "102/102 [==============================] - 11s 104ms/step - loss: 0.0293\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.494068153864542 - normalized_discounted_cumulative_gain@5(0.0): 0.5605928322073352 - mean_average_precision(0.0): 0.5139849024134501\n",
      "Epoch 14/20\n",
      "102/102 [==============================] - 9s 90ms/step - loss: 0.0279\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.4987296535086411 - normalized_discounted_cumulative_gain@5(0.0): 0.5671151052462753 - mean_average_precision(0.0): 0.5215687290635513\n",
      "Epoch 15/20\n",
      "102/102 [==============================] - 10s 95ms/step - loss: 0.0250\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5098806065082092 - normalized_discounted_cumulative_gain@5(0.0): 0.5740280134728454 - mean_average_precision(0.0): 0.5333708142044251\n",
      "Epoch 16/20\n",
      "102/102 [==============================] - 11s 108ms/step - loss: 0.0182\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5153947436725983 - normalized_discounted_cumulative_gain@5(0.0): 0.5762000806926193 - mean_average_precision(0.0): 0.5298641458868614\n",
      "Epoch 17/20\n",
      "102/102 [==============================] - 11s 108ms/step - loss: 0.0178\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5134634304473057 - normalized_discounted_cumulative_gain@5(0.0): 0.56980318132021 - mean_average_precision(0.0): 0.5282068879797845\n",
      "Epoch 18/20\n",
      "102/102 [==============================] - 11s 110ms/step - loss: 0.0145\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5176147404760021 - normalized_discounted_cumulative_gain@5(0.0): 0.581247024416785 - mean_average_precision(0.0): 0.5377934773692993\n",
      "Epoch 19/20\n",
      "102/102 [==============================] - 9s 92ms/step - loss: 0.0172\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.49962649604962145 - normalized_discounted_cumulative_gain@5(0.0): 0.56731906503144 - mean_average_precision(0.0): 0.5253291982119823\n",
      "Epoch 20/20\n",
      "102/102 [==============================] - 9s 92ms/step - loss: 0.0181\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5087417209853161 - normalized_discounted_cumulative_gain@5(0.0): 0.5729943933841338 - mean_average_precision(0.0): 0.5320246681539154\n"
     ]
    }
   ],
   "source": [
    "history = model.fit_generator(train_generator, epochs=20, callbacks=[evaluate], workers=30, use_multiprocessing=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{normalized_discounted_cumulative_gain@3(0.0): 0.5087417209853161,\n",
       " normalized_discounted_cumulative_gain@5(0.0): 0.5729943933841338,\n",
       " mean_average_precision(0.0): 0.5320246681539154}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate(test_x, test_y, batch_size=5)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "Use this function to update the README.md with a better set of parameters.\n",
    "Make sure you delete the correct section of the README.md before calling this function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "append_params_to_readme(model)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
